{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "# Welcome to JARVIS.\n",
        "\n",
        "# The core of JARVIS is powered by Open Interpreter:\n",
        "\n",
        "for chunk in interpreter.chat(\"What's 34/24?\", stream=True, display=False):\n",
        "  print(chunk)\n",
        "\n",
        "# (This cell is for demonstration purposes. Do not run it until you've setup JARVIS below.)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I-16BvnEun7n",
        "outputId": "77ada0ad-e6a3-4f6b-a36c-372924f1f0bc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'role': 'assistant', 'type': 'code', 'format': 'python', 'start': True}\n",
            "{'role': 'assistant', 'type': 'code', 'format': 'python', 'content': '34'}\n",
            "{'role': 'assistant', 'type': 'code', 'format': 'python', 'content': '/'}\n",
            "{'role': 'assistant', 'type': 'code', 'format': 'python', 'content': '24'}\n",
            "{'role': 'assistant', 'type': 'code', 'format': 'python', 'end': True}\n",
            "{'role': 'computer', 'type': 'confirmation', 'format': 'execution', 'content': {'type': 'code', 'format': 'python', 'content': '34/24'}}\n",
            "{'role': 'computer', 'type': 'console', 'start': True}\n",
            "{'role': 'computer', 'type': 'console', 'format': 'active_line', 'content': 1}\n",
            "{'role': 'computer', 'type': 'console', 'format': 'output', 'content': '1.4166666666666667'}\n",
            "{'role': 'computer', 'type': 'console', 'format': 'active_line', 'content': None}\n",
            "{'role': 'computer', 'type': 'console', 'end': True}\n",
            "{'role': 'assistant', 'type': 'message', 'start': True}\n",
            "{'role': 'assistant', 'type': 'message', 'content': 'Approx'}\n",
            "{'role': 'assistant', 'type': 'message', 'content': 'imately'}\n",
            "{'role': 'assistant', 'type': 'message', 'content': ' '}\n",
            "{'role': 'assistant', 'type': 'message', 'content': '1'}\n",
            "{'role': 'assistant', 'type': 'message', 'content': '.'}\n",
            "{'role': 'assistant', 'type': 'message', 'content': '417'}\n",
            "{'role': 'assistant', 'type': 'message', 'content': '.'}\n",
            "{'role': 'assistant', 'type': 'message', 'end': True}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Install **(You must run ❗️`Runtime > Restart Session`❗️ after this)**"
      ],
      "metadata": {
        "id": "lE2GSFLJoOtI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install open-interpreter"
      ],
      "metadata": {
        "id": "iwI1Dhv7uzD-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "05188b29-5bea-4967-cdac-8d7e7e319b8f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: open-interpreter in /usr/local/lib/python3.10/dist-packages (0.2.0)\n",
            "Requirement already satisfied: appdirs<2.0.0,>=1.4.4 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (1.4.4)\n",
            "Requirement already satisfied: astor<0.9.0,>=0.8.1 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (0.8.1)\n",
            "Requirement already satisfied: git-python<2.0.0,>=1.0.3 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (1.0.3)\n",
            "Requirement already satisfied: html2image<3.0.0.0,>=2.0.4.3 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (2.0.4.3)\n",
            "Requirement already satisfied: inquirer<4.0.0,>=3.1.3 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (3.2.1)\n",
            "Requirement already satisfied: ipykernel<7.0.0,>=6.26.0 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (6.28.0)\n",
            "Requirement already satisfied: jupyter-client<9.0.0,>=8.6.0 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (8.6.0)\n",
            "Requirement already satisfied: litellm<2.0.0,>=1.15.10 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (1.16.19)\n",
            "Requirement already satisfied: matplotlib<4.0.0,>=3.8.2 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (3.8.2)\n",
            "Requirement already satisfied: openai<2.0.0,>=1.6.1 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (1.6.1)\n",
            "Requirement already satisfied: posthog<4.0.0,>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (3.1.0)\n",
            "Requirement already satisfied: psutil<6.0.0,>=5.9.6 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (5.9.7)\n",
            "Requirement already satisfied: pyyaml<7.0.0,>=6.0.1 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (6.0.1)\n",
            "Requirement already satisfied: rich<14.0.0,>=13.4.2 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (13.7.0)\n",
            "Requirement already satisfied: six<2.0.0,>=1.16.0 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (1.16.0)\n",
            "Requirement already satisfied: tiktoken<0.5.0,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (0.4.0)\n",
            "Requirement already satisfied: tokentrim<0.2.0,>=0.1.13 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (0.1.13)\n",
            "Requirement already satisfied: toml<0.11.0,>=0.10.2 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (0.10.2)\n",
            "Requirement already satisfied: wget<4.0,>=3.2 in /usr/local/lib/python3.10/dist-packages (from open-interpreter) (3.2)\n",
            "Requirement already satisfied: gitpython in /usr/local/lib/python3.10/dist-packages (from git-python<2.0.0,>=1.0.3->open-interpreter) (3.1.40)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from html2image<3.0.0.0,>=2.0.4.3->open-interpreter) (2.31.0)\n",
            "Requirement already satisfied: websocket-client<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from html2image<3.0.0.0,>=2.0.4.3->open-interpreter) (1.7.0)\n",
            "Requirement already satisfied: blessed>=1.19.0 in /usr/local/lib/python3.10/dist-packages (from inquirer<4.0.0,>=3.1.3->open-interpreter) (1.20.0)\n",
            "Requirement already satisfied: editor>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from inquirer<4.0.0,>=3.1.3->open-interpreter) (1.6.5)\n",
            "Requirement already satisfied: readchar>=3.0.6 in /usr/local/lib/python3.10/dist-packages (from inquirer<4.0.0,>=3.1.3->open-interpreter) (4.0.5)\n",
            "Requirement already satisfied: comm>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel<7.0.0,>=6.26.0->open-interpreter) (0.2.1)\n",
            "Requirement already satisfied: debugpy>=1.6.5 in /usr/local/lib/python3.10/dist-packages (from ipykernel<7.0.0,>=6.26.0->open-interpreter) (1.6.6)\n",
            "Requirement already satisfied: ipython>=7.23.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel<7.0.0,>=6.26.0->open-interpreter) (7.34.0)\n",
            "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /usr/local/lib/python3.10/dist-packages (from ipykernel<7.0.0,>=6.26.0->open-interpreter) (5.7.0)\n",
            "Requirement already satisfied: matplotlib-inline>=0.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel<7.0.0,>=6.26.0->open-interpreter) (0.1.6)\n",
            "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.10/dist-packages (from ipykernel<7.0.0,>=6.26.0->open-interpreter) (1.5.8)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from ipykernel<7.0.0,>=6.26.0->open-interpreter) (23.2)\n",
            "Requirement already satisfied: pyzmq>=24 in /usr/local/lib/python3.10/dist-packages (from ipykernel<7.0.0,>=6.26.0->open-interpreter) (25.1.2)\n",
            "Requirement already satisfied: tornado>=6.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel<7.0.0,>=6.26.0->open-interpreter) (6.3.2)\n",
            "Requirement already satisfied: traitlets>=5.4.0 in /usr/local/lib/python3.10/dist-packages (from ipykernel<7.0.0,>=6.26.0->open-interpreter) (5.7.1)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from jupyter-client<9.0.0,>=8.6.0->open-interpreter) (2.8.2)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from litellm<2.0.0,>=1.15.10->open-interpreter) (3.9.1)\n",
            "Requirement already satisfied: certifi<2024.0.0,>=2023.7.22 in /usr/local/lib/python3.10/dist-packages (from litellm<2.0.0,>=1.15.10->open-interpreter) (2023.11.17)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from litellm<2.0.0,>=1.15.10->open-interpreter) (8.1.7)\n",
            "Requirement already satisfied: importlib-metadata>=6.8.0 in /usr/local/lib/python3.10/dist-packages (from litellm<2.0.0,>=1.15.10->open-interpreter) (7.0.1)\n",
            "Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /usr/local/lib/python3.10/dist-packages (from litellm<2.0.0,>=1.15.10->open-interpreter) (3.1.2)\n",
            "Requirement already satisfied: python-dotenv>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from litellm<2.0.0,>=1.15.10->open-interpreter) (1.0.0)\n",
            "Requirement already satisfied: tokenizers in /usr/local/lib/python3.10/dist-packages (from litellm<2.0.0,>=1.15.10->open-interpreter) (0.15.0)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4.0.0,>=3.8.2->open-interpreter) (1.2.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4.0.0,>=3.8.2->open-interpreter) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4.0.0,>=3.8.2->open-interpreter) (4.47.0)\n",
            "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4.0.0,>=3.8.2->open-interpreter) (1.4.5)\n",
            "Requirement already satisfied: numpy<2,>=1.21 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4.0.0,>=3.8.2->open-interpreter) (1.23.5)\n",
            "Requirement already satisfied: pillow>=8 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4.0.0,>=3.8.2->open-interpreter) (9.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4.0.0,>=3.8.2->open-interpreter) (3.1.1)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.6.1->open-interpreter) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai<2.0.0,>=1.6.1->open-interpreter) (1.7.0)\n",
            "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.6.1->open-interpreter) (0.26.0)\n",
            "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.6.1->open-interpreter) (2.5.3)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.6.1->open-interpreter) (1.3.0)\n",
            "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.6.1->open-interpreter) (4.66.1)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.6.1->open-interpreter) (4.9.0)\n",
            "Requirement already satisfied: monotonic>=1.5 in /usr/local/lib/python3.10/dist-packages (from posthog<4.0.0,>=3.1.0->open-interpreter) (1.6)\n",
            "Requirement already satisfied: backoff>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from posthog<4.0.0,>=3.1.0->open-interpreter) (2.2.1)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=13.4.2->open-interpreter) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=13.4.2->open-interpreter) (2.16.1)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->open-interpreter) (2023.6.3)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.6.1->open-interpreter) (3.6)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.6.1->open-interpreter) (1.2.0)\n",
            "Requirement already satisfied: wcwidth>=0.1.4 in /usr/local/lib/python3.10/dist-packages (from blessed>=1.19.0->inquirer<4.0.0,>=3.1.3->open-interpreter) (0.2.12)\n",
            "Requirement already satisfied: runs in /usr/local/lib/python3.10/dist-packages (from editor>=1.6.0->inquirer<4.0.0,>=3.1.3->open-interpreter) (1.2.0)\n",
            "Requirement already satisfied: xmod in /usr/local/lib/python3.10/dist-packages (from editor>=1.6.0->inquirer<4.0.0,>=3.1.3->open-interpreter) (1.8.1)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.6.1->open-interpreter) (1.0.2)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai<2.0.0,>=1.6.1->open-interpreter) (0.14.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata>=6.8.0->litellm<2.0.0,>=1.15.10->open-interpreter) (3.17.0)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel<7.0.0,>=6.26.0->open-interpreter) (67.7.2)\n",
            "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel<7.0.0,>=6.26.0->open-interpreter) (0.19.1)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel<7.0.0,>=6.26.0->open-interpreter) (4.4.2)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel<7.0.0,>=6.26.0->open-interpreter) (0.7.5)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel<7.0.0,>=6.26.0->open-interpreter) (3.0.43)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel<7.0.0,>=6.26.0->open-interpreter) (0.2.0)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.23.1->ipykernel<7.0.0,>=6.26.0->open-interpreter) (4.9.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2<4.0.0,>=3.1.2->litellm<2.0.0,>=1.15.10->open-interpreter) (2.1.3)\n",
            "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.10/dist-packages (from jupyter-core!=5.0.*,>=4.12->ipykernel<7.0.0,>=6.26.0->open-interpreter) (4.1.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=13.4.2->open-interpreter) (0.1.2)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai<2.0.0,>=1.6.1->open-interpreter) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.14.6 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai<2.0.0,>=1.6.1->open-interpreter) (2.14.6)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->html2image<3.0.0.0,>=2.0.4.3->open-interpreter) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->html2image<3.0.0.0,>=2.0.4.3->open-interpreter) (2.0.7)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->litellm<2.0.0,>=1.15.10->open-interpreter) (23.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->litellm<2.0.0,>=1.15.10->open-interpreter) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->litellm<2.0.0,>=1.15.10->open-interpreter) (1.9.4)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->litellm<2.0.0,>=1.15.10->open-interpreter) (1.4.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->litellm<2.0.0,>=1.15.10->open-interpreter) (1.3.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->litellm<2.0.0,>=1.15.10->open-interpreter) (4.0.3)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from gitpython->git-python<2.0.0,>=1.0.3->open-interpreter) (4.0.11)\n",
            "Requirement already satisfied: huggingface_hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers->litellm<2.0.0,>=1.15.10->open-interpreter) (0.20.1)\n",
            "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from gitdb<5,>=4.0.1->gitpython->git-python<2.0.0,>=1.0.3->open-interpreter) (5.0.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers->litellm<2.0.0,>=1.15.10->open-interpreter) (3.13.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers->litellm<2.0.0,>=1.15.10->open-interpreter) (2023.6.0)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /usr/local/lib/python3.10/dist-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel<7.0.0,>=6.26.0->open-interpreter) (0.8.3)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/dist-packages (from pexpect>4.3->ipython>=7.23.1->ipykernel<7.0.0,>=6.26.0->open-interpreter) (0.7.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/openai/whisper.git -q\n",
        "!pip install gradio==3.50 -q\n",
        "!pip install elevenlabs -q"
      ],
      "metadata": {
        "id": "GIyJZlbVYob4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "619e0903-eae5-4d8c-a203-4d584fce03c5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.3/20.3 MB\u001b[0m \u001b[31m66.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m299.2/299.2 kB\u001b[0m \u001b[31m34.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Set your API Keys"
      ],
      "metadata": {
        "id": "yBlPE7TRVJWF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "eleven_labs_api_key = \"<your_api_key>\" # https://elevenlabs.io/speech-synthesis\n",
        "openai_api_key = \"<your_api_key>\" # https://platform.openai.com/account/api-keys"
      ],
      "metadata": {
        "id": "LI_6uNbs_K9W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Setup"
      ],
      "metadata": {
        "id": "aeic06e-o5Wh"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Misc Imports"
      ],
      "metadata": {
        "id": "CjrhRX6fWkXL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import gradio as gr\n",
        "import time"
      ],
      "metadata": {
        "id": "XbzTmzACWlyV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Open Interpreter"
      ],
      "metadata": {
        "id": "T9KaJXLXQtYJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from interpreter import interpreter\n",
        "\n",
        "interpreter.llm.api_key = openai_api_key\n",
        "interpreter.auto_run = True"
      ],
      "metadata": {
        "id": "gpNOy1sLQs0v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Whisper"
      ],
      "metadata": {
        "id": "TQ9iTzMQYs9u"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import whisper\n",
        "model = whisper.load_model(\"base\")"
      ],
      "metadata": {
        "id": "YstqtPbGoWXA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def transcribe(audio):\n",
        "\n",
        "    # load audio and pad/trim it to fit 30 seconds\n",
        "    audio = whisper.load_audio(audio)\n",
        "    audio = whisper.pad_or_trim(audio)\n",
        "\n",
        "    # make log-Mel spectrogram and move to the same device as the model\n",
        "    mel = whisper.log_mel_spectrogram(audio).to(model.device)\n",
        "\n",
        "    # detect the spoken language\n",
        "    _, probs = model.detect_language(mel)\n",
        "\n",
        "    # decode the audio\n",
        "    options = whisper.DecodingOptions()\n",
        "    result = whisper.decode(model, mel, options)\n",
        "    return result.text"
      ],
      "metadata": {
        "id": "JtTvvQQPcOZZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### ElevenLabs"
      ],
      "metadata": {
        "id": "kf751XxlyOd9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from elevenlabs import generate, play, set_api_key\n",
        "\n",
        "set_api_key(eleven_labs_api_key)"
      ],
      "metadata": {
        "id": "qRcI6nlx8Cun"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import io\n",
        "from pydub import AudioSegment\n",
        "\n",
        "def get_audio_length(audio_bytes):\n",
        "  # Create a BytesIO object from the byte array\n",
        "  byte_io = io.BytesIO(audio_bytes)\n",
        "\n",
        "  # Load the audio data with PyDub\n",
        "  audio = AudioSegment.from_mp3(byte_io)\n",
        "\n",
        "  # Get the length of the audio in milliseconds\n",
        "  length_ms = len(audio)\n",
        "\n",
        "  # Optionally convert to seconds\n",
        "  length_s = length_ms / 1000.0\n",
        "\n",
        "  return length_s"
      ],
      "metadata": {
        "id": "o78_YmQwEBvL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def speak(text):\n",
        "  speaking = True\n",
        "  audio = generate(\n",
        "      text=text,\n",
        "      voice=\"Daniel\"\n",
        "  )\n",
        "  play(audio, notebook=True)\n",
        "\n",
        "  audio_length = get_audio_length(audio)\n",
        "  time.sleep(audio_length)"
      ],
      "metadata": {
        "id": "Ru3Z4M_L_FCK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Run"
      ],
      "metadata": {
        "id": "X93f9Q5E_Gd5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# @title JARVIS\n",
        "# @markdown ### **Setup Instructions**\n",
        "# @markdown 1. Run this cell, then scroll down to use the interface (don't click the link, and **give the interface 60 seconds to load**).\n",
        "# @markdown 2. Press the `Record from Microphone` button.\n",
        "# @markdown 3. Allow access to your microphone, then speak your command.\n",
        "# @markdown 4. Stop the recording, then press `Submit`.\n",
        "# @markdown\n",
        "# @markdown\n",
        "# @markdown JARVIS will respond verbally + carry out your command.\n",
        "\n",
        "last_sentence = \"\"\n",
        "\n",
        "with gr.Blocks() as demo:\n",
        "\n",
        "    chatbot = gr.Chatbot()\n",
        "    audio_input = gr.inputs.Audio(source=\"microphone\", type=\"filepath\")\n",
        "    btn = gr.Button(\"Submit\")\n",
        "\n",
        "    def transcribe(audio):\n",
        "      audio = whisper.load_audio(audio)\n",
        "      audio = whisper.pad_or_trim(audio)\n",
        "      mel = whisper.log_mel_spectrogram(audio).to(model.device)\n",
        "      _, probs = model.detect_language(mel)\n",
        "      options = whisper.DecodingOptions()\n",
        "      result = whisper.decode(model, mel, options)\n",
        "      return result.text\n",
        "\n",
        "    def add_user_message(audio, history):\n",
        "        user_message = transcribe(audio)\n",
        "        return history + [[user_message, None]]\n",
        "\n",
        "    def bot(history):\n",
        "        global last_sentence\n",
        "\n",
        "        user_message = history[-1][0]\n",
        "        history[-1][1] = \"\"\n",
        "        active_block_type = \"\"\n",
        "        language = \"\"\n",
        "        for chunk in interpreter.chat(user_message, stream=True, display=False):\n",
        "\n",
        "            # I built this before we build the flags, like \"start\": True and \"end\": True.\n",
        "            # See the streaming example above. You can use those \"start\" and \"end\" flags to\n",
        "            # start the code blocks, message blocks, etc. Here we track it manually and ignore the flags.\n",
        "\n",
        "            # You should use the flags though! I was just lazy. We should rebuild this soon.\n",
        "\n",
        "            # Message\n",
        "            if chunk[\"type\"] == \"message\" and \"content\" in chunk:\n",
        "              if active_block_type != \"message\":\n",
        "                active_block_type = \"message\"\n",
        "              history[-1][1] += chunk[\"content\"]\n",
        "\n",
        "              last_sentence += chunk[\"content\"]\n",
        "              if any([punct in last_sentence for punct in \".?!\\n\"]):\n",
        "                yield history\n",
        "                speak(last_sentence)\n",
        "                last_sentence = \"\"\n",
        "              else:\n",
        "                yield history\n",
        "\n",
        "            # Code\n",
        "            if chunk[\"type\"] == \"code\" and \"content\" in chunk:\n",
        "              if active_block_type != \"code\":\n",
        "                active_block_type = \"code\"\n",
        "                history[-1][1] += f\"\\n```{chunk['format']}\"\n",
        "              history[-1][1] += chunk[\"content\"]\n",
        "              yield history\n",
        "\n",
        "            # Output\n",
        "            if chunk[\"type\"] == \"confirmation\":\n",
        "              history[-1][1] += \"\\n```\\n\\n```text\\n\"\n",
        "              yield history\n",
        "            if chunk[\"type\"] == \"console\":\n",
        "              if chunk.get(\"format\") == \"output\":\n",
        "                if chunk[\"content\"] == \"KeyboardInterrupt\":\n",
        "                  break\n",
        "                history[-1][1] += chunk[\"content\"] + \"\\n\"\n",
        "                yield history\n",
        "              if chunk.get(\"format\") == \"active_line\" and chunk[\"content\"] == None:\n",
        "                # Active line will be none when we finish execution.\n",
        "                # You could also detect this with \"type\": \"console\", \"end\": True.\n",
        "                history[-1][1] = history[-1][1].strip()\n",
        "                history[-1][1] += \"\\n```\\n\"\n",
        "                yield history\n",
        "\n",
        "        if last_sentence:\n",
        "          speak(last_sentence)\n",
        "\n",
        "    btn.click(add_user_message, [audio_input, chatbot], [chatbot]).then(\n",
        "        bot, chatbot, chatbot\n",
        "    )\n",
        "\n",
        "demo.queue()\n",
        "demo.launch(debug=True)"
      ],
      "metadata": {
        "id": "O-xIJaH949uv",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Text-only JARVIS\n",
        "# @markdown Run this cell for a ChatGPT-like interface.\n",
        "\n",
        "with gr.Blocks() as demo:\n",
        "    chatbot = gr.Chatbot()\n",
        "    msg = gr.Textbox()\n",
        "\n",
        "    def user(user_message, history):\n",
        "        return \"\", history + [[user_message, None]]\n",
        "\n",
        "    def bot(history):\n",
        "\n",
        "        user_message = history[-1][0]\n",
        "        history[-1][1] = \"\"\n",
        "        active_block_type = \"\"\n",
        "\n",
        "        for chunk in interpreter.chat(user_message, stream=True, display=False):\n",
        "\n",
        "            # Message\n",
        "            if chunk[\"type\"] == \"message\" and \"content\" in chunk:\n",
        "              if active_block_type != \"message\":\n",
        "                active_block_type = \"message\"\n",
        "              history[-1][1] += chunk[\"content\"]\n",
        "\n",
        "              last_sentence += chunk[\"content\"]\n",
        "              if any([punct in last_sentence for punct in \".?!\\n\"]):\n",
        "                yield history\n",
        "                speak(last_sentence)\n",
        "                last_sentence = \"\"\n",
        "              else:\n",
        "                yield history\n",
        "\n",
        "            # Code\n",
        "            if chunk[\"type\"] == \"code\" and \"content\" in chunk:\n",
        "              if active_block_type != \"code\":\n",
        "                active_block_type = \"code\"\n",
        "                history[-1][1] += f\"\\n```{chunk['format']}\"\n",
        "              history[-1][1] += chunk[\"content\"]\n",
        "              yield history\n",
        "\n",
        "            # Output\n",
        "            if chunk[\"type\"] == \"confirmation\":\n",
        "              history[-1][1] += \"\\n```\\n\\n```text\\n\"\n",
        "              yield history\n",
        "            if chunk[\"type\"] == \"console\":\n",
        "              if chunk.get(\"format\") == \"output\":\n",
        "                if chunk[\"content\"] == \"KeyboardInterrupt\":\n",
        "                  break\n",
        "                history[-1][1] += chunk[\"content\"] + \"\\n\"\n",
        "                yield history\n",
        "              if chunk.get(\"format\") == \"active_line\" and chunk[\"content\"] == None:\n",
        "                # Active line will be none when we finish execution.\n",
        "                # You could also detect this with \"type\": \"console\", \"end\": True.\n",
        "                history[-1][1] = history[-1][1].strip()\n",
        "                history[-1][1] += \"\\n```\\n\"\n",
        "                yield history\n",
        "\n",
        "    msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(\n",
        "        bot, chatbot, chatbot\n",
        "    )\n",
        "\n",
        "demo.queue()\n",
        "demo.launch(debug=True)"
      ],
      "metadata": {
        "id": "mL1LS3NTlTtv",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "lE2GSFLJoOtI",
        "yBlPE7TRVJWF",
        "aeic06e-o5Wh",
        "CjrhRX6fWkXL",
        "T9KaJXLXQtYJ",
        "TQ9iTzMQYs9u",
        "kf751XxlyOd9"
      ],
      "gpuType": "T4"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.9"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}